package com.eascs.spider.ant.worker.spider;

import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.impl.client.CloseableHttpClient;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.eascs.spider.ant.common.proj.Platform;
import com.eascs.spider.ant.common.proj.Task;
import com.eascs.spider.ant.common.util.NetUtil;
import com.eascs.spider.ant.common.util.PropertiesUtil;

/**
 * 新浪微博爬虫类
 * @author Administrator
 *
 */
public class SinaWeiboSpider extends SimpleSpider{
	
	private String cookie ;
	
	public SinaWeiboSpider(String url) {
		super(url);
		this.cookie = PropertiesUtil.getConfig("sina_weibo_cookie");
	}

	@Override
	public void run() {
		Map<String, String> headerMap = new HashMap<String, String>();
		headerMap.put("Host", "weibo.com");
		headerMap.put("Cookie", cookie);
		String html = NetUtil.loadPageRawTextByGet(url, client, headerMap , null, "utf-8");
		if(html == null){
			logger.warn("html为空,url:{}" ,url);
			return ;
		}
		extractUrl(html);
		if(!url.equals("https://weibo.com/"))
			extractWeiboDetail(html);
		else
			loadMore();
			
	}

	//从html中提取符合条件的微博url
	private void extractUrl(String html){
		html = html.replaceAll("\\\\", "");
		Pattern pattern = Pattern.compile("href=\"(//weibo.com/(\\d+)/.*?)\"");
		Matcher matcher = pattern.matcher(html);
		while(matcher.find()){
			String urlTemp = "https:" + matcher.group(1);
			Task task = new Task();
			task.setUrl(urlTemp);
			manager.addSendQueue(task);
		}
	}
	
	//从单条微博中提取详情
	private void extractWeiboDetail(String html){
		Pattern pattern = Pattern.compile("<script>FM.view(.*?)</script>");
		Matcher matcher = pattern.matcher(html);
		Map<String, Object> data = new HashMap<>();
		while(matcher.find()){
			try {
				if(matcher.group().contains("pl.content.weiboDetail.index")){
					Pattern p2 = Pattern.compile("\"html\":\"(.*?)\"}");
					Matcher m2 = p2.matcher(matcher.group());
					if(m2.find()){
						String htmlTemp = m2.group(1).replaceAll("\\\\n", "").replaceAll("\\\\", "");
						Document document = Jsoup.parse(htmlTemp);
						String blogger = document.select("div.WB_detail div.WB_info").text();
						logger.info("博主：{}" , blogger);
						String blog_time = document.select("div.WB_detail div[class=WB_from S_txt2]").text();
						if(blog_time.contains("来自")){
							logger.info("发博时间：{},来源:{}" ,blog_time.split("来自")[0],blog_time.split("来自")[1]);
							data.put("time", blog_time.split("来自")[0]);
							data.put("source", blog_time.split("来自")[1]);
						}else{
							logger.info("发博时间：{}" ,blog_time);
							data.put("time", blog_time);
						}
						String blog_content = document.select("div.WB_detail div[class=WB_text W_f14]").text();
						logger.info("博文内容:{}" ,blog_content);
						data.put("blogger", blogger);
						data.put("content", blog_content);
						data.put("url", url);
						data.put("crawler_time", new Date(System.currentTimeMillis()));
						mongoSession.setData(data, Platform.SINA_WEIBO);
					}
				}
			} catch (Exception e) {
				logger.error("提取详情发生异常,url：{}" , url, e);
			}
		}
	}

	//请求首页之后默认再ajax加载10页
	private void loadMore(){
		String url = "https://weibo.com/a/aj/transform/loadingmoreunlogin";
		CloseableHttpClient client = NetUtil.initHttpClient();
		Map<String, String> header = new HashMap<>();
		header.put("Host", "weibo.com");
		header.put("Cookie", "YF-Ugrow-G0=56862bac2f6bf97368b95873bc687eef; SUB=_2AkMtRKrNf8NxqwJRmP4WxWrnbIV2wgDEieKbGFsWJRMxHRl-yT83qnwMtRB6XNyh2GPnMpD86whk3_WR8AY0foCMOU5U; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WFYAuhHOin2KrCYsJoaLyu9; login_sid_t=0c57fc365b7f781d284f165968f653f8; cross_origin_proto=SSL; YF-V5-G0=b8115b96b42d4782ab3a2201c5eba25d; _s_tentry=passport.weibo.com; UOR=passport.weibo.com,weibo.com,spr_sinamkt_buy_hyww_weibo_t126; Apache=1410511079246.2239.1511532375742; SINAGLOBAL=1410511079246.2239.1511532375742; ULV=1511532375756:1:1:1:1410511079246.2239.1511532375742:");
		header.put("X-Requested-With", "XMLHttpRequest");
		Map<String, String> param = new HashMap<>();
		param.put("ajwvr", "6");
		param.put("category", "0");
		param.put("lefnav", "0");
		
		for(int i=2;i<10;i++){
			logger.info("微博首页ajax加载第：{}页" ,i);
			param.put("__rnd", ""+System.currentTimeMillis());
			param.put("page", "" + i);
			String htmlTemp = NetUtil.loadPageRawTextByGet(url, client, header, param, "utf-8");
			extractUrl(htmlTemp);
			sleep(1000);
		}
	}
	
	private void sleep(long mesc){
		try {
			Thread.sleep(mesc);
		} catch (Exception e) {
			// TODO: handle exception
		}
	}
	
}
